
import pandas as pd
import numpy as np
import requests as rq
import tweepy as tp
from tweepy import OAuthHandler
import json
import os
from timeit import default_timer as timer
import black
from bs4 import BeautifulSoup
import re
from IPython.display import Image
from matplotlib import pyplot as plt
import seaborn as sns


def get_url_elements(row):
    soup = BeautifulSoup(row["source"], "html")
    row["tweet_source"] = soup.find("a").contents[0]
    row["url_ref"] = soup.find("a").get("href")
    row["rel"] = soup.find("a").get("rel")[0]
    return row


df_werated = pd.read_csv(os.path.join(os.getcwd(), "twitter-archive-enhanced.csv"))
df_werated.shape

(2356, 17)


file_url = "https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv"
r = rq.get(file_url)


with open(os.path.join(os.getcwd(), file_url.split("/")[-1]), "wb") as file:
    file.write(r.content)


df_img_pred = pd.read_csv("image-predictions.tsv", sep="\t")


consumer_key = "HIDDEN"
consumer_secret = "HIDDEN"
access_token = "HIDDEN"
access_secret = "HIDDEN"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tp.API(auth, wait_on_rate_limit=True)


tweet_ids = df_werated.tweet_id.values
len(tweet_ids)

2356


count = 0
fails_dict = {}
start = timer()
with open("tweet_json.txt", "w") as outfile:
    for tweet_id in tweet_ids:
        count += 1

        try:
            tweet = api.get_status(tweet_id, tweet_mode="extended")
            json.dump(tweet._json, outfile)
            outfile.write("\n")
        except tp.TweepyException as e:
            fails_dict[tweet_id] = e
            pass
end = timer()
print(f"Total elapsed time: {end - start} seconds")

Total elapsed time: 710.6608249999999 seconds


file_url = "https://video.udacity-data.com/topher/2018/November/5be5fb7d_tweet-json/tweet-json.txt"
r = rq.get(file_url)
with open(os.path.join(os.getcwd(), file_url.split("/")[-1]), "wb") as file:
    file.write(r.content)


with open("tweet-json.txt", "r") as j:
    json_1st_line = json.loads(j.readlines(1)[0])


json_1st_line

{'created_at': 'Tue Aug 01 16:23:56 +0000 2017',
 'id': 892420643555336193,
 'id_str': '892420643555336193',
 'full_text': "This is Phineas. He's a mystical boy. Only ever appears in the hole of a donut. 13/10 https://t.co/MgUWQ76dJU",
 'truncated': False,
 'display_text_range': [0, 85],
 'entities': {'hashtags': [],
  'symbols': [],
  'user_mentions': [],
  'urls': [],
  'media': [{'id': 892420639486877696,
    'id_str': '892420639486877696',
    'indices': [86, 109],
    'media_url': 'http://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg',
    'media_url_https': 'https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg',
    'url': 'https://t.co/MgUWQ76dJU',
    'display_url': 'pic.twitter.com/MgUWQ76dJU',
    'expanded_url': 'https://twitter.com/dog_rates/status/892420643555336193/photo/1',
    'type': 'photo',
    'sizes': {'large': {'w': 540, 'h': 528, 'resize': 'fit'},
     'thumb': {'w': 150, 'h': 150, 'resize': 'crop'},
     'small': {'w': 540, 'h': 528, 'resize': 'fit'},
     'medium': {'w': 540, 'h': 528, 'resize': 'fit'}}}]},
 'extended_entities': {'media': [{'id': 892420639486877696,
    'id_str': '892420639486877696',
    'indices': [86, 109],
    'media_url': 'http://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg',
    'media_url_https': 'https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg',
    'url': 'https://t.co/MgUWQ76dJU',
    'display_url': 'pic.twitter.com/MgUWQ76dJU',
    'expanded_url': 'https://twitter.com/dog_rates/status/892420643555336193/photo/1',
    'type': 'photo',
    'sizes': {'large': {'w': 540, 'h': 528, 'resize': 'fit'},
     'thumb': {'w': 150, 'h': 150, 'resize': 'crop'},
     'small': {'w': 540, 'h': 528, 'resize': 'fit'},
     'medium': {'w': 540, 'h': 528, 'resize': 'fit'}}}]},
 'source': '<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>',
 'in_reply_to_status_id': None,
 'in_reply_to_status_id_str': None,
 'in_reply_to_user_id': None,
 'in_reply_to_user_id_str': None,
 'in_reply_to_screen_name': None,
 'user': {'id': 4196983835,
  'id_str': '4196983835',
  'name': 'WeRateDogs™ (author)',
  'screen_name': 'dog_rates',
  'location': 'DM YOUR DOGS, WE WILL RATE',
  'description': '#1 Source for Professional Dog Ratings | STORE: @ShopWeRateDogs | IG, FB & SC: WeRateDogs MOBILE APP: @GoodDogsGame | Business: dogratingtwitter@gmail.com',
  'url': 'https://t.co/N7sNNHAEXS',
  'entities': {'url': {'urls': [{'url': 'https://t.co/N7sNNHAEXS',
      'expanded_url': 'http://weratedogs.com',
      'display_url': 'weratedogs.com',
      'indices': [0, 23]}]},
   'description': {'urls': []}},
  'protected': False,
  'followers_count': 3200889,
  'friends_count': 104,
  'listed_count': 2784,
  'created_at': 'Sun Nov 15 21:41:29 +0000 2015',
  'favourites_count': 114031,
  'utc_offset': None,
  'time_zone': None,
  'geo_enabled': True,
  'verified': True,
  'statuses_count': 5288,
  'lang': 'en',
  'contributors_enabled': False,
  'is_translator': False,
  'is_translation_enabled': False,
  'profile_background_color': '000000',
  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',
  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',
  'profile_background_tile': False,
  'profile_image_url': 'http://pbs.twimg.com/profile_images/861415328504569856/R2xOOfwe_normal.jpg',
  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/861415328504569856/R2xOOfwe_normal.jpg',
  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/4196983835/1501129017',
  'profile_link_color': 'F5ABB5',
  'profile_sidebar_border_color': '000000',
  'profile_sidebar_fill_color': '000000',
  'profile_text_color': '000000',
  'profile_use_background_image': False,
  'has_extended_profile': True,
  'default_profile': False,
  'default_profile_image': False,
  'following': True,
  'follow_request_sent': False,
  'notifications': False,
  'translator_type': 'none'},
 'geo': None,
 'coordinates': None,
 'place': None,
 'contributors': None,
 'is_quote_status': False,
 'retweet_count': 8853,
 'favorite_count': 39467,
 'favorited': False,
 'retweeted': False,
 'possibly_sensitive': False,
 'possibly_sensitive_appealable': False,
 'lang': 'en'}


json_1st_line["entities"]["media"][0]["expanded_url"]

'https://twitter.com/dog_rates/status/892420643555336193/photo/1'


# Checking if all the columns can be found.
all(elem in json_1st_line.keys() for elem in ["id", "retweet_count", "favorite_count"])

True


tweets_dict = {
    "tweet_id": [],
    "retweet_count": [],
    "favorite_count": [],
    "followers_count": [],
    "friends_count": [],
    "source": [],
    "retweeted_status": [],
}
with open("tweet-json.txt", "r") as j:
    for l in j.readlines():
        r = json.loads(l)
        tweets_dict["tweet_id"].append(r["id"])
        tweets_dict["retweet_count"].append(r["retweet_count"])
        tweets_dict["favorite_count"].append(r["favorite_count"])
        tweets_dict["followers_count"].append(r["user"]["followers_count"])
        tweets_dict["friends_count"].append(r["user"]["friends_count"])
        tweets_dict["source"].append(r["source"])
        rt_status = r.get("retweeted_status")
        if not rt_status:
            rt_status = "Original Tweet"
        else:
            rt_status = "Retweet"
        tweets_dict["retweeted_status"].append(rt_status)


df_tweet_details = pd.DataFrame(tweets_dict)


pd.set_option("display.max_rows", None)
pd.set_option("display.max_columns", None)


df_werated


df_img_pred


df_tweet_details


# Returning df view to standard in order to save memory
pd.reset_option("all", silent=True)


df_werated.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2356 entries, 0 to 2355
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype  
---  ------                      --------------  -----  
 0   tweet_id                    2356 non-null   int64  
 1   in_reply_to_status_id       78 non-null     float64
 2   in_reply_to_user_id         78 non-null     float64
 3   timestamp                   2356 non-null   object 
 4   source                      2356 non-null   object 
 5   text                        2356 non-null   object 
 6   retweeted_status_id         181 non-null    float64
 7   retweeted_status_user_id    181 non-null    float64
 8   retweeted_status_timestamp  181 non-null    object 
 9   expanded_urls               2297 non-null   object 
 10  rating_numerator            2356 non-null   int64  
 11  rating_denominator          2356 non-null   int64  
 12  name                        2356 non-null   object 
 13  doggo                       2356 non-null   object 
 14  floofer                     2356 non-null   object 
 15  pupper                      2356 non-null   object 
 16  puppo                       2356 non-null   object 
dtypes: float64(4), int64(3), object(10)
memory usage: 313.0+ KB


# Looking for duplicated values
df_werated.tweet_id.duplicated().sum()

0


# Checking the profile of the columns that should remain numeric
df_werated[["rating_numerator", "rating_denominator"]].describe()


df_werated.query("rating_denominator > 10").text.shape[0]

20


for i in df_werated.query("rating_denominator > 10").index:
    print(f"row: {i}", "-->", df_werated.text[i])

row: 342 --> @docmisterio account started on 11/15/15
row: 433 --> The floofs have been released I repeat the floofs have been released. 84/70 https://t.co/NIYC820tmd
row: 784 --> RT @dog_rates: After so many requests, this is Bretagne. She was the last surviving 9/11 search dog, and our second ever 14/10. RIP https:/…
row: 902 --> Why does this never happen at my front door... 165/150 https://t.co/HmwrdfEfUE
row: 1068 --> After so many requests, this is Bretagne. She was the last surviving 9/11 search dog, and our second ever 14/10. RIP https://t.co/XAVDNDaVgQ
row: 1120 --> Say hello to this unbelievably well behaved squad of doggos. 204/170 would try to pet all at once https://t.co/yGQI3He3xv
row: 1165 --> Happy 4/20 from the squad! 13/10 for all https://t.co/eV1diwds8a
row: 1202 --> This is Bluebert. He just saw that both #FinalFur match ups are split 50/50. Amazed af. 11/10 https://t.co/Kky1DPG4iq
row: 1228 --> Happy Saturday here's 9 puppers on a bench. 99/90 good work everybody https://t.co/mpvaVxKmc1
row: 1254 --> Here's a brigade of puppers. All look very prepared for whatever happens next. 80/80 https://t.co/0eb7R1Om12
row: 1274 --> From left to right:
Cletus, Jerome, Alejandro, Burp, &amp; Titson
None know where camera is. 45/50 would hug all at once https://t.co/sedre1ivTK
row: 1351 --> Here is a whole flock of puppers.  60/50 I'll take the lot https://t.co/9dpcw6MdWa
row: 1433 --> Happy Wednesday here's a bucket of pups. 44/40 would pet all at once https://t.co/HppvrYuamZ
row: 1598 --> Yes I do realize a rating of 4/20 would've been fitting. However, it would be unjust to give these cooperative pups that low of a rating
row: 1634 --> Two sneaky puppers were not initially seen, moving the rating to 143/130. Please forgive us. Thank you https://t.co/kRK51Y5ac3
row: 1635 --> Someone help the girl is being mugged. Several are distracting her while two steal her shoes. Clever puppers 121/110 https://t.co/1zfnTJLt55
row: 1662 --> This is Darrel. He just robbed a 7/11 and is in a high speed police chase. Was just spotted by the helicopter 10/10 https://t.co/7EsP8LmSp5
row: 1663 --> I'm aware that I could've said 20/16, but here at WeRateDogs we are very professional. An inconsistent rating scale is simply irresponsible
row: 1779 --> IT'S PUPPERGEDDON. Total of 144/120 ...I think https://t.co/ZanVtAtvIq
row: 1843 --> Here we have an entire platoon of puppers. Total score: 88/80 would pet all at once https://t.co/y93p6FLvVw


df_werated.iloc[[784, 1202, 1662], :][
    ["tweet_id", "rating_numerator", "rating_denominator"]
]


df_tweets_to_correct = pd.DataFrame(
    {
        "tweet_id": ["775096608509886464", "716439118184652801", "682962037429899265"],
        "new_numerator": [14, 11, 10],
        "new_denominator": [10, 10, 10],
    }
)


df_werated.rating_numerator.value_counts()

12      558
11      464
10      461
13      351
9       158
8       102
7        55
14       54
5        37
6        32
3        19
4        17
2         9
1         9
75        2
15        2
420       2
0         2
80        1
144       1
17        1
26        1
20        1
121       1
143       1
44        1
60        1
45        1
50        1
99        1
204       1
1776      1
165       1
666       1
27        1
182       1
24        1
960       1
84        1
88        1
Name: rating_numerator, dtype: int64


df_werated.query("rating_numerator >= 100").text.shape[0]

11


df_werated.query("rating_numerator < 10").text.shape[0]

440


for i in df_werated.query("rating_numerator >= 100").index:
    print(f"row: {i}", "-->", df_werated.text[i], "--", df_werated.rating_numerator[i])

row: 188 --> @dhmontgomery We also gave snoop dogg a 420/10 but I think that predated your research -- 420
row: 189 --> @s8n You tried very hard to portray this good boy as not so good, but you have ultimately failed. His goodness shines through. 666/10 -- 666
row: 290 --> @markhoppus 182/10 -- 182
row: 313 --> @jonnysun @Lin_Manuel ok jomny I know you're excited but 960/00 isn't a valid rating, 13/10 is tho -- 960
row: 902 --> Why does this never happen at my front door... 165/150 https://t.co/HmwrdfEfUE -- 165
row: 979 --> This is Atticus. He's quite simply America af. 1776/10 https://t.co/GRXwMxLBkh -- 1776
row: 1120 --> Say hello to this unbelievably well behaved squad of doggos. 204/170 would try to pet all at once https://t.co/yGQI3He3xv -- 204
row: 1634 --> Two sneaky puppers were not initially seen, moving the rating to 143/130. Please forgive us. Thank you https://t.co/kRK51Y5ac3 -- 143
row: 1635 --> Someone help the girl is being mugged. Several are distracting her while two steal her shoes. Clever puppers 121/110 https://t.co/1zfnTJLt55 -- 121
row: 1779 --> IT'S PUPPERGEDDON. Total of 144/120 ...I think https://t.co/ZanVtAtvIq -- 144
row: 2074 --> After so many requests... here you go.

Good dogg. 420/10 https://t.co/yfAAo1gdeY -- 420


# Checking if there is decimal numerators in the text

df_werated.text.str.findall("\d+\.\d+\/\d+").sum()

['13.5/10', '9.75/10', '9.75/10', '11.27/10', '9.5/10', '11.26/10']


# Checking if there is decimal denominators in the text as well
df_werated.text.str.findall("\d+/\d+\.\d+").sum()

[]


# The fun of this site is on numerators above 10. Let's get a sample to see how they look like in the text
for i in df_werated.query("rating_numerator < 10").sample(5).index:
    try:
        id = df_werated.tweet_id[i]
        url = df_img_pred.query("tweet_id == @id").jpg_url.values[0]
        print(f"row: {i}", "-->", df_werated.text[i])
        display(Image(url, width=150, height=150))
    except:
        print(f"row: {i}", "-->", df_werated.text[i], "\n", "No Image to display")

row: 1734 --> This pup's name is Sabertooth (parents must be cool). Ears for days. Jumps unannounced. 9/10 would pet diligently https://t.co/iazoiNUviP

row: 1837 --> "Yes hello I'ma just snag this here toasted bagel real quick. carry on." 9/10 https://t.co/Cuz0Osnekp

row: 936 --> This is Hank. He's mischievous af. Doesn't even know what he was trying to do here. 8/10 quit the shit Hank damn https://t.co/3r7wjfsXHc

row: 1378 --> This is Yoda. He's a Zimbabwean Rutabaga. Freaks out if u stop scratching his belly. Incredibly self-centered. 9/10 https://t.co/yVdMsVYHIx

row: 485 --> RT @dog_rates: Meet Beau &amp; Wilbur. Wilbur stole Beau's bed from him. Wilbur now has so much room for activities. 9/10 for both pups https:/… 
 No Image to display


# Checking for suspicious names
df_werated.name.value_counts().head(20)

None       745
a           55
Charlie     12
Cooper      11
Lucy        11
Oliver      11
Tucker      10
Penny       10
Lola        10
Winston      9
Bo           9
Sadie        8
the          8
Daisy        7
Buddy        7
Toby         7
an           7
Bailey       7
Leo          6
Oscar        6
Name: name, dtype: int64


set(df_werated[~df_werated.name.str[0].str.isupper()].name)

{'a',
 'actually',
 'all',
 'an',
 'by',
 'getting',
 'his',
 'incredibly',
 'infuriating',
 'just',
 'life',
 'light',
 'mad',
 'my',
 'not',
 'officially',
 'old',
 'one',
 'quite',
 'space',
 'such',
 'the',
 'this',
 'unacceptable',
 'very'}


# Checking the column source
for e in df_werated.source.sample(3):
    print(e)

<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>
<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>
<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>


df_img_pred.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2075 entries, 0 to 2074
Data columns (total 12 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   tweet_id  2075 non-null   int64  
 1   jpg_url   2075 non-null   object 
 2   img_num   2075 non-null   int64  
 3   p1        2075 non-null   object 
 4   p1_conf   2075 non-null   float64
 5   p1_dog    2075 non-null   bool   
 6   p2        2075 non-null   object 
 7   p2_conf   2075 non-null   float64
 8   p2_dog    2075 non-null   bool   
 9   p3        2075 non-null   object 
 10  p3_conf   2075 non-null   float64
 11  p3_dog    2075 non-null   bool   
dtypes: bool(3), float64(3), int64(2), object(4)
memory usage: 152.1+ KB


df_img_pred.describe()


df_img_pred.p1.value_counts().head(20)

golden_retriever             150
Labrador_retriever           100
Pembroke                      89
Chihuahua                     83
pug                           57
chow                          44
Samoyed                       43
toy_poodle                    39
Pomeranian                    38
cocker_spaniel                30
malamute                      30
French_bulldog                26
miniature_pinscher            23
Chesapeake_Bay_retriever      23
seat_belt                     22
Siberian_husky                20
German_shepherd               20
Staffordshire_bullterrier     20
Cardigan                      19
web_site                      19
Name: p1, dtype: int64


df_img_pred.p2.value_counts().head(20)

Labrador_retriever                104
golden_retriever                   92
Cardigan                           73
Chihuahua                          44
Pomeranian                         42
Chesapeake_Bay_retriever           41
French_bulldog                     41
toy_poodle                         37
cocker_spaniel                     34
miniature_poodle                   33
Siberian_husky                     33
beagle                             28
Pembroke                           27
Eskimo_dog                         27
collie                             27
kuvasz                             26
Italian_greyhound                  22
American_Staffordshire_terrier     21
Pekinese                           21
chow                               20
Name: p2, dtype: int64


df_img_pred.p3.value_counts().head(20)

Labrador_retriever                79
Chihuahua                         58
golden_retriever                  48
Eskimo_dog                        38
kelpie                            35
kuvasz                            34
Staffordshire_bullterrier         32
chow                              32
beagle                            31
cocker_spaniel                    31
Pomeranian                        29
toy_poodle                        29
Pekinese                          29
Chesapeake_Bay_retriever          27
Great_Pyrenees                    27
Pembroke                          27
French_bulldog                    26
malamute                          26
American_Staffordshire_terrier    24
pug                               23
Name: p3, dtype: int64


df_img_pred.tweet_id.duplicated().sum()

0


df_tweet_details.head()


df_tweet_details.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2354 entries, 0 to 2353
Data columns (total 7 columns):
 #   Column            Non-Null Count  Dtype 
---  ------            --------------  ----- 
 0   tweet_id          2354 non-null   int64 
 1   retweet_count     2354 non-null   int64 
 2   favorite_count    2354 non-null   int64 
 3   followers_count   2354 non-null   int64 
 4   friends_count     2354 non-null   int64 
 5   source            2354 non-null   object
 6   retweeted_status  2354 non-null   object
dtypes: int64(5), object(2)
memory usage: 128.9+ KB


df_tweet_details.describe()


df_tweet_details.tweet_id.duplicated().sum()

0


df_werated_clean = df_werated.copy()
df_img_pred_clean = df_img_pred.copy()
df_tweet_details_clean = df_tweet_details.copy()


# Dealing with string columns
df_werated_clean["tweet_id"] = df_werated_clean["tweet_id"].astype(str)
df_img_pred_clean["tweet_id"] = df_img_pred_clean["tweet_id"].astype(str)
df_tweet_details_clean["tweet_id"] = df_tweet_details_clean["tweet_id"].astype(str)


df_werated_clean = df_werated_clean.fillna(
    value={
        "in_reply_to_status_id": 0.0,
        "in_reply_to_user_id": 0.0,
        "retweeted_status_id": 0.0,
        "retweeted_status_user_id": 0.0,
    },
    method=None,
)


df_werated_clean = df_werated_clean.astype(
    {
        "in_reply_to_status_id": "int64",
        "in_reply_to_user_id": "int64",
        "retweeted_status_id": "int64",
        "retweeted_status_user_id": "int64",
    }
)


df_werated_clean = df_werated_clean.astype(
    {
        "in_reply_to_status_id": "str",
        "in_reply_to_user_id": "str",
        "retweeted_status_id": "str",
        "retweeted_status_user_id": "str",
    }
)


cols = [
    "in_reply_to_status_id",
    "in_reply_to_user_id",
    "retweeted_status_id",
    "retweeted_status_user_id",
]
for c in cols:
    df_werated_clean[c] = df_werated_clean[c].replace({"0": np.NaN})


# Dealing with date and time columns
df_werated_clean["timestamp"] = pd.to_datetime(
    df_werated_clean["timestamp"]
).dt.tz_convert(None)


df_werated_clean.dtypes

tweet_id                              object
in_reply_to_status_id                 object
in_reply_to_user_id                   object
timestamp                     datetime64[ns]
source                                object
text                                  object
retweeted_status_id                   object
retweeted_status_user_id              object
retweeted_status_timestamp            object
expanded_urls                         object
rating_numerator                       int64
rating_denominator                     int64
name                                  object
doggo                                 object
floofer                               object
pupper                                object
puppo                                 object
dtype: object


df_img_pred_clean.tweet_id.dtypes

dtype('O')


df_tweet_details_clean.tweet_id.dtypes

dtype('O')


df_werated_clean = df_werated_clean[df_werated_clean.retweeted_status_id.isnull()]


df_tweet_details_clean = df_tweet_details_clean.query(
    "retweeted_status == 'Original Tweet'"
)


df_werated_clean.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 2175 entries, 0 to 2355
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype         
---  ------                      --------------  -----         
 0   tweet_id                    2175 non-null   object        
 1   in_reply_to_status_id       78 non-null     object        
 2   in_reply_to_user_id         78 non-null     object        
 3   timestamp                   2175 non-null   datetime64[ns]
 4   source                      2175 non-null   object        
 5   text                        2175 non-null   object        
 6   retweeted_status_id         0 non-null      object        
 7   retweeted_status_user_id    0 non-null      object        
 8   retweeted_status_timestamp  0 non-null      object        
 9   expanded_urls               2117 non-null   object        
 10  rating_numerator            2175 non-null   int64         
 11  rating_denominator          2175 non-null   int64         
 12  name                        2175 non-null   object        
 13  doggo                       2175 non-null   object        
 14  floofer                     2175 non-null   object        
 15  pupper                      2175 non-null   object        
 16  puppo                       2175 non-null   object        
dtypes: datetime64[ns](1), int64(2), object(14)
memory usage: 305.9+ KB


df_tweet_details_clean.retweeted_status.value_counts()

Original Tweet    2175
Name: retweeted_status, dtype: int64


df_werated_clean[df_werated_clean["expanded_urls"].isnull()]


df_werated_clean[df_werated_clean["expanded_urls"].isnull()].info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 58 entries, 30 to 2298
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype         
---  ------                      --------------  -----         
 0   tweet_id                    58 non-null     object        
 1   in_reply_to_status_id       55 non-null     object        
 2   in_reply_to_user_id         55 non-null     object        
 3   timestamp                   58 non-null     datetime64[ns]
 4   source                      58 non-null     object        
 5   text                        58 non-null     object        
 6   retweeted_status_id         0 non-null      object        
 7   retweeted_status_user_id    0 non-null      object        
 8   retweeted_status_timestamp  0 non-null      object        
 9   expanded_urls               0 non-null      object        
 10  rating_numerator            58 non-null     int64         
 11  rating_denominator          58 non-null     int64         
 12  name                        58 non-null     object        
 13  doggo                       58 non-null     object        
 14  floofer                     58 non-null     object        
 15  pupper                      58 non-null     object        
 16  puppo                       58 non-null     object        
dtypes: datetime64[ns](1), int64(2), object(14)
memory usage: 8.2+ KB


df_miss_expanded_url = (
    df_werated_clean[df_werated_clean["expanded_urls"].isnull()]
    .copy()
    .reset_index(drop=True)
)


for i in df_miss_expanded_url.index:
    print(df_miss_expanded_url.tweet_id[i], "-->", df_miss_expanded_url.text[i])

886267009285017600 --> @NonWhiteHat @MayhewMayhem omg hello tanner you are a scary good boy 12/10 would pet with extreme caution
881633300179243008 --> @roushfenway These are good dogs but 17/10 is an emotional impulse rating. More like 13/10s
879674319642796034 --> @RealKentMurphy 14/10 confirmed
870726314365509632 --> @ComplicitOwl @ShopWeRateDogs &gt;10/10 is reserved for dogs
863427515083354112 --> @Jack_Septic_Eye I'd need a few more pics to polish a full analysis, but based on the good boy content above I'm leaning towards 12/10
857214891891077121 --> @Marc_IRL pixelated af 12/10
856288084350160898 --> @xianmcguire @Jenna_Marbles Kardashians wouldn't be famous if as a society we didn't place enormous value on what they do. The dogs are very deserving of their 14/10
855862651834028034 --> @dhmontgomery We also gave snoop dogg a 420/10 but I think that predated your research
855860136149123072 --> @s8n You tried very hard to portray this good boy as not so good, but you have ultimately failed. His goodness shines through. 666/10
850333567704068097 --> @markhoppus MARK THAT DOG HAS SEEN AND EXPERIENCED MANY THINGS. PROBABLY LOST OTHER EAR DOING SOMETHING HEROIC. 13/10 HUG THE DOG HOPPUS
848213670039564288 --> Jerry just apuppologized to me. He said there was no ill-intent to the slippage. I overreacted I admit. Pupgraded to an 11/10 would pet
847617282490613760 --> .@breaannanicolee PUPDATE: Cannon has a heart on his nose. Pupgraded to a 13/10
840698636975636481 --> @0_kelvin_0 &gt;10/10 is reserved for puppos sorry Kevin
838150277551247360 --> @markhoppus 182/10
838085839343206401 --> @bragg6of8 @Andy_Pace_ we are still looking for the first 15/10
835246439529840640 --> @jonnysun @Lin_Manuel ok jomny I know you're excited but 960/00 isn't a valid rating, 13/10 is tho
832088576586297345 --> @docmisterio account started on 11/15/15
831926988323639298 --> @UNC can confirm 12/10
828361771580813312 --> Beebop and Doobert should start a band 12/10 would listen
826598799820865537 --> I was going to do 007/10, but the joke wasn't worth the &lt;10 rating
823333489516937216 --> @HistoryInPics 13/10
821153421864615936 --> @imgur for a polar bear tho I'd say 13/10 is appropriate
813130366689148928 --> I've been informed by multiple sources that this is actually a dog elf who's tired from helping Santa all night. Pupgraded to 12/10
811647686436880384 --> PUPDATE: I've been informed that Augie was actually bringing his family these flowers when he tripped. Very good boy. Pupgraded to 11/10
801854953262350336 --> .@NBCSports OMG THE TINY HAT I'M GOING TO HAVE TO SAY 11/10 NBC
800859414831898624 --> @SkyWilliams doggo simply protecting you from evil that which you cannot see. 11/10 would give extra pets
797165961484890113 --> @JODYHiGHROLLER it may be an 11/10 but what do I know 😉
786051337297522688 --> 13/10 for breakdancing puppo @shibbnbot
785515384317313025 --> Today, 10/10, should be National Dog Rates Day
766714921925144576 --> His name is Charley and he already has a new set of wheels thanks to donations. I heard his top speed was also increased. 13/10 for Charley
763956972077010945 --> @TheEllenShow I'm not sure if you know this but that doggo right there is a 12/10
750381685133418496 --> 13/10 such a good doggo
@spaghemily
747651430853525504 --> Other pupper asked not to have his identity shared. Probably just embarrassed about the headbutt. Also 12/10 it'll be ok mystery pup
738891149612572673 --> @mount_alex3 13/10
707983188426153984 --> @serial @MrRoles OH MY GOD I listened to all of season 1 during a single road trip. I love you guys! I can confirm Bernie's 12/10 rating :)
704491224099647488 --> 13/10 hero af
@ABC
696518437233913856 --> Oh my god 10/10 for every little hot dog pupper
696490539101908992 --> After reading the comments I may have overestimated this pup. Downgraded to a 1/10. Please forgive me
693644216740769793 --> BREAKING PUPDATE: I've just been notified that (if in U.S.) this dog appears to be operating the vehicle. Upgraded to 10/10. Skilled af
693582294167244802 --> Personally I'd give him an 11/10. Not sure why you think you're qualified to rate such a stellar pup.
@CommonWhiteGirI
692423280028966913 --> PUPDATE: just noticed this dog has some extra legs. Very advanced. Revolutionary af. Upgraded to a 9/10
690607260360429569 --> 12/10 @LightningHoltt
686035780142297088 --> Yes I do realize a rating of 4/20 would've been fitting. However, it would be unjust to give these cooperative pups that low of a rating
685681090388975616 --> Jack deserves another round of applause. If you missed this earlier today I strongly suggest reading it. Wonderful first 14/10 🐶❤️
684969860808454144 --> For those who claim this is a goat, u are wrong. It is not the Greatest Of All Time. The rating of 5/10 should have made that clear. Thank u
682808988178739200 --> I'm aware that I could've said 20/16, but here at WeRateDogs we are very professional. An inconsistent rating scale is simply irresponsible
681340665377193984 --> I've been told there's a slight possibility he's checking his mirror. We'll bump to 9.5/10. Still a menace
678023323247357953 --> After getting lost in Reese's eyes for several minutes we're going to upgrade him to a 13/10
676590572941893632 --> After some outrage from the crowd. Bubbles is being upgraded to a 7/10. That's as high as I'm going. Thank you
675849018447167488 --> This dog is being demoted to a 9/10 for not wearing a helmet while riding. Gotta stay safe out there. Thank you
674742531037511680 --> Some clarification is required. The dog is singing Cher and that is more than worthy of an 11/10. Thank you
674606911342424069 --> The 13/10 also takes into account this impeccable yard. Louis is great but the future dad in me can't ignore that luscious green grass
674330906434379776 --> 13/10
@ABC7
673716320723169284 --> The millennials have spoken and we've decided to immediately demote to a 1/10. Thank you
671550332464455680 --> After 22 minutes of careful deliberation this dog is being demoted to a 1/10. The longer you look at him the more terrifying he becomes
669684865554620416 --> After countless hours of research and hundreds of formula alterations we have concluded that Dug should be bumped to an 11/10
668967877119254528 --> 12/10 good shit Bubka
@wane15
667070482143944705 --> After much debate this dog is being upgraded to 10/10. I repeat 10/10


# Checking if there is an actual prediction for these tweets:
df_img_pred_clean[df_img_pred_clean.tweet_id.isin(list(df_miss_expanded_url.tweet_id))]


df_werated_clean = df_werated_clean.query("expanded_urls == expanded_urls")


df_werated_clean.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 2117 entries, 0 to 2355
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype         
---  ------                      --------------  -----         
 0   tweet_id                    2117 non-null   object        
 1   in_reply_to_status_id       23 non-null     object        
 2   in_reply_to_user_id         23 non-null     object        
 3   timestamp                   2117 non-null   datetime64[ns]
 4   source                      2117 non-null   object        
 5   text                        2117 non-null   object        
 6   retweeted_status_id         0 non-null      object        
 7   retweeted_status_user_id    0 non-null      object        
 8   retweeted_status_timestamp  0 non-null      object        
 9   expanded_urls               2117 non-null   object        
 10  rating_numerator            2117 non-null   int64         
 11  rating_denominator          2117 non-null   int64         
 12  name                        2117 non-null   object        
 13  doggo                       2117 non-null   object        
 14  floofer                     2117 non-null   object        
 15  pupper                      2117 non-null   object        
 16  puppo                       2117 non-null   object        
dtypes: datetime64[ns](1), int64(2), object(14)
memory usage: 297.7+ KB


df_img_pred_clean[
    df_img_pred_clean.tweet_id.isin(
        list(
            df_werated_clean[
                ~df_werated_clean["in_reply_to_status_id"].isnull()
            ].tweet_id
        )
    )
].shape

(23, 12)


df_werated_clean[
    ~df_werated_clean["in_reply_to_status_id"].isnull()
].in_reply_to_user_id.value_counts()

4196983835    23
Name: in_reply_to_user_id, dtype: int64


# Prooving that the texts are of an image rating
df_rep_tweet = df_werated_clean[~df_werated_clean["in_reply_to_user_id"].isnull()]
for i in df_rep_tweet.index:
    print(df_rep_tweet.tweet_id[i], "-->", df_rep_tweet.text[i])

863079547188785154 --> Ladies and gentlemen... I found Pipsy. He may have changed his name to Pablo, but he never changed his love for the sea. Pupgraded to 14/10 https://t.co/lVU5GyNFen
856526610513747968 --> THIS IS CHARLIE, MARK. HE DID JUST WANT TO SAY HI AFTER ALL. PUPGRADED TO A 14/10. WOULD BE AN HONOR TO FLY WITH https://t.co/p1hBHCmWnA
844979544864018432 --> PUPDATE: I'm proud to announce that Toby is 236 days sober. Pupgraded to a 13/10. We're all very proud of you, Toby https://t.co/a5OaJeRl9B
802265048156610565 --> Like doggo, like pupper version 2. Both 11/10 https://t.co/9IxWAXFqze
746906459439529985 --> PUPDATE: can't see any. Even if I could, I couldn't reach them to pet. 0/10 much disappointment https://t.co/c7WXaB2nqX
746818907684614144 --> Guys... Dog Jesus 2.0
13/10 buoyant af https://t.co/CuNA7OwfKQ
729838605770891264 --> "Challenge completed" 
(pupgraded to 12/10) https://t.co/85dTK7XCXB
705786532653883392 --> Seriously, add us 🐶 11/10 for sad wet pupper https://t.co/xwPE9faVZR
704871453724954624 --> I found a forest Pipsy. 12/10 https://t.co/mIQ1KoVsmU
703425003149250560 --> Really guys? Again? I know this is a rare Albanian Bingo Seal, but we only rate dogs. Only send in dogs... 9/10 https://t.co/6JYLpUmBrC
695767669421768709 --> If you are aware of who is making these please let me know. 13/10 vroom vroom https://t.co/U0D1sbIDrG
694356675654983680 --> This pupper only appears through the hole of a Funyun. Much like Phineas, this one is also mysterious af. 10/10 https://t.co/SQsEBWxPyG
692142790915014657 --> These are some pictures of Teddy that further justify his 13/10 rating. Please enjoy https://t.co/tDkJAnQsbQ
684538444857667585 --> After watching this video, we've determined that Pippa will be upgraded to a 12/10. Please enjoy https://t.co/IKoRK4yoxV
684225744407494656 --> Two sneaky puppers were not initially seen, moving the rating to 143/130. Please forgive us. Thank you https://t.co/kRK51Y5ac3
675870721063669760 --> &amp; this is Yoshi. Another world record contender 11/10 (what the hell is happening why are there so many contenders?) https://t.co/QG708dDNH6
675707330206547968 --> We've got ourselves a battle here. Watch out Reggie. 11/10 https://t.co/ALJvbtcwf0
675349384339542016 --> Yea I lied. Here's more. All 13/10 https://t.co/ZQZf2U4xCP
674999807681908736 --> Ok last one of these. I may try to make some myself. Anyway here ya go. 13/10 https://t.co/i9CDd1oEu8
674793399141146624 --> I have found another. 13/10 https://t.co/HwroPYv8pY
674754018082705410 --> Just received another perfect photo of dogs and the sunset. 12/10 https://t.co/9YmNcxA2Cc
671729906628341761 --> I'm just going to leave this one here as well. 13/10 https://t.co/DaD5SyajWt
669353438988365824 --> This is Tessa. She is also very pleased after finally meeting her biological father. 10/10 https://t.co/qDS1aCqppv


df_werated_clean.name = df_werated_clean.name.str.replace("^[a-z]+", "None")

C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: The default value of regex will change from True to False in a future version.
  """Entry point for launching an IPython kernel.


df_werated_clean.name.value_counts().head(20)

None       726
Lucy        11
Charlie     11
Cooper      10
Oliver      10
Tucker       9
Penny        9
Winston      8
Lola         8
Sadie        8
Daisy        7
Toby         7
Stanley      6
Koda         6
Oscar        6
Jax          6
Bella        6
Bailey       6
Bo           6
Rusty        5
Name: name, dtype: int64


df_werated_clean.name.sample(10)

1198         Link
426      Sundance
1907         None
1048         Kilo
1676         None
651       Monster
226          None
2047    Scruffers
261       Charlie
1233      Stubert
Name: name, dtype: object


df_tweets_to_correct


for i, tweet in enumerate(df_tweets_to_correct.tweet_id):

    r = df_werated_clean.query("tweet_id == @tweet")
    if len(r) > 0:
        df_werated_clean["rating_numerator"][r.index] = df_tweets_to_correct[
            "new_numerator"
        ][i]
        df_werated_clean["rating_denominator"][r.index] = df_tweets_to_correct[
            "new_denominator"
        ][i]

C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  import sys
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  # Remove the CWD from sys.path while we load stuff.


df_werated_clean = df_werated_clean.reset_index(drop=True)


df_werated_clean[df_werated_clean["tweet_id"].isin(df_tweets_to_correct["tweet_id"])][
    ["tweet_id", "rating_numerator", "rating_denominator"]
]


# First, we need to change the data type of these fields
df_werated_clean["rating_numerator"] = df_werated_clean["rating_numerator"].astype(
    float
)


df_werated_clean["rating_denominator"] = df_werated_clean["rating_denominator"].astype(
    float
)


df_werated_clean.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2117 entries, 0 to 2116
Data columns (total 17 columns):
 #   Column                      Non-Null Count  Dtype         
---  ------                      --------------  -----         
 0   tweet_id                    2117 non-null   object        
 1   in_reply_to_status_id       23 non-null     object        
 2   in_reply_to_user_id         23 non-null     object        
 3   timestamp                   2117 non-null   datetime64[ns]
 4   source                      2117 non-null   object        
 5   text                        2117 non-null   object        
 6   retweeted_status_id         0 non-null      object        
 7   retweeted_status_user_id    0 non-null      object        
 8   retweeted_status_timestamp  0 non-null      object        
 9   expanded_urls               2117 non-null   object        
 10  rating_numerator            2117 non-null   float64       
 11  rating_denominator          2117 non-null   float64       
 12  name                        2117 non-null   object        
 13  doggo                       2117 non-null   object        
 14  floofer                     2117 non-null   object        
 15  pupper                      2117 non-null   object        
 16  puppo                       2117 non-null   object        
dtypes: datetime64[ns](1), float64(2), object(14)
memory usage: 281.3+ KB


df_werated_clean.reset_index(drop=True, inplace=True)


dic_ratings = {"text": [], "index": [], "value": []}


for i, t in enumerate(df_werated_clean["text"]):
    if re.search("\d+\.\d+\/\d+", t):
        dic_ratings["text"].append(t)
        dic_ratings["index"].append(i)
        dic_ratings["value"].append(re.search("\d+\.\d+", t).group())


for i, v in enumerate(dic_ratings["index"]):
    df_werated_clean["rating_numerator"][v] = dic_ratings["value"][i]

C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


for t in df_werated_clean.query("rating_numerator>= rating_numerator.std()")["text"]:
    print(t)

The floofs have been released I repeat the floofs have been released. 84/70 https://t.co/NIYC820tmd
Why does this never happen at my front door... 165/150 https://t.co/HmwrdfEfUE
This is Atticus. He's quite simply America af. 1776/10 https://t.co/GRXwMxLBkh
Say hello to this unbelievably well behaved squad of doggos. 204/170 would try to pet all at once https://t.co/yGQI3He3xv
Happy Saturday here's 9 puppers on a bench. 99/90 good work everybody https://t.co/mpvaVxKmc1
Here's a brigade of puppers. All look very prepared for whatever happens next. 80/80 https://t.co/0eb7R1Om12
From left to right:
Cletus, Jerome, Alejandro, Burp, &amp; Titson
None know where camera is. 45/50 would hug all at once https://t.co/sedre1ivTK
Here is a whole flock of puppers.  60/50 I'll take the lot https://t.co/9dpcw6MdWa
Happy Wednesday here's a bucket of pups. 44/40 would pet all at once https://t.co/HppvrYuamZ
Two sneaky puppers were not initially seen, moving the rating to 143/130. Please forgive us. Thank you https://t.co/kRK51Y5ac3
Someone help the girl is being mugged. Several are distracting her while two steal her shoes. Clever puppers 121/110 https://t.co/1zfnTJLt55
IT'S PUPPERGEDDON. Total of 144/120 ...I think https://t.co/ZanVtAtvIq
Here we have an entire platoon of puppers. Total score: 88/80 would pet all at once https://t.co/y93p6FLvVw
After so many requests... here you go.

Good dogg. 420/10 https://t.co/yfAAo1gdeY


df_werated_clean["rating"] = (
    df_werated_clean["rating_numerator"] / df_werated_clean["rating_denominator"]
)


for i in dic_ratings["index"]:
    print(
        df_werated_clean["rating_numerator"][i],
        "-->",
        df_werated_clean["text"][i][
            : re.search("\d+\.\d+", df_werated_clean["text"][i]).end() + 1
        ],
    )

13.5 --> This is Bella. She hopes her smile made you smile. If not, she is also offering you her favorite monkey. 13.5/
9.75 --> This is Logan, the Chow who lived. He solemnly swears he's up to lots of good. H*ckin magical af 9.75/
11.27 --> This is Sophie. She's a Jubilant Bush Pupper. Super h*ckin rare. Appears at random just to smile at the locals. 11.27/
11.26 --> Here we have uncovered an entire battalion of holiday puppers. Average of 11.26/


df_werated_clean[
    ["tweet_id", "rating_numerator", "rating_denominator", "rating"]
].sample(20)


df_werated_clean.describe()


# Number of unique breeds before
bb1 = df_img_pred_clean.p1.unique()
bb2 = df_img_pred_clean.p2.unique()
bb3 = df_img_pred_clean.p3.unique()


df_img_pred_clean[["p1", "p2", "p3"]] = df_img_pred_clean[["p1", "p2", "p3"]].replace(
    "_", " ", regex=True
)
df_img_pred_clean[["p1", "p2", "p3"]] = df_img_pred_clean[["p1", "p2", "p3"]].apply(
    lambda col: col.str.title()
)


ba1 = df_img_pred_clean.p1.unique()
ba2 = df_img_pred_clean.p2.unique()
ba3 = df_img_pred_clean.p3.unique()


print(
    " p1 changed:",
    len(bb1) != len(ba1),
    "\n",
    "p2 changed:",
    len(bb2) != len(ba2),
    "\n",
    "p3 changed:",
    len(bb1) != len(ba1),
)

 p1 changed: False 
 p2 changed: True 
 p3 changed: False


df_img_pred_clean[["p1", "p2", "p3"]].sample(5)


df_img_pred_clean["pred_dog_breed"] = None
df_img_pred_clean["pred_conf"] = np.NaN


for i, row in df_img_pred_clean.iterrows():
    if row.p1_dog:
        df_img_pred_clean["pred_dog_breed"][i] = row.p1
        df_img_pred_clean["pred_conf"][i] = row.p1_conf
    elif row.p2_dog:
        df_img_pred_clean["pred_dog_breed"][i] = row.p2
        df_img_pred_clean["pred_conf"][i] = row.p2_conf
    elif row.p3_dog:
        df_img_pred_clean["pred_dog_breed"][i] = row.p3
        df_img_pred_clean["pred_conf"][i] = row.p3_conf
    else:
        df_img_pred_clean["pred_dog_breed"][i] = "Not a Dog"
        df_img_pred_clean["pred_conf"][i] = np.NaN

C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  This is separate from the ipykernel package so we can avoid doing imports until
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  after removing the cwd from sys.path.
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:12: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  if sys.path[0] == "":
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:13: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  del sys.path[0]
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:9: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  if __name__ == "__main__":
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  # Remove the CWD from sys.path while we load stuff.
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
C:\Users\iurkl\AppData\Local\Programs\Python\Python37\lib\site-packages\ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  import sys


df_img_pred_clean.sample(10)


df_img_pred_clean.pred_dog_breed.value_counts()

Not a Dog               324
Golden Retriever        173
Labrador Retriever      113
Pembroke                 96
Chihuahua                95
                       ... 
Scotch Terrier            1
Entlebucher               1
Japanese Spaniel          1
Standard Schnauzer        1
Bouvier Des Flandres      1
Name: pred_dog_breed, Length: 114, dtype: int64


df_img_pred_clean.query("pred_dog_breed == 'Not a Dog'").sample(10)


for i in df_img_pred_clean.query("pred_dog_breed == 'Not a Dog'").sample(10).index:
    try:

        url = df_img_pred_clean.jpg_url[i]
        print(
            f"p1:{df_img_pred_clean.p1[i]}",
            f"p2:{df_img_pred_clean.p2[i]}",
            f"p3:{df_img_pred_clean.p3[i]}",
        )
        display(Image(url, width=150, height=150))
    except:
        print(
            f"p1:{df_img_pred_clean.p1[i]}",
            f"p2:{df_img_pred_clean.p2[i]}",
            f"p3:{df_img_pred_clean.p3[i]}",
            "\n",
            "No Image to display",
        )

p1:Ibex p2:Bighorn p3:Ram

p1:Marmot p2:Meerkat p3:Weasel

p1:Coil p2:Dugong p3:Rain Barrel

p1:Killer Whale p2:Grey Whale p3:Hammerhead

p1:Lawn Mower p2:Swing p3:Barrow

p1:Vacuum p2:Swab p3:Toilet Tissue

p1:Seashore p2:Promontory p3:Wreck

p1:Ostrich p2:Bearskin p3:Swab

p1:Otter p2:Mink p3:Sea Lion

p1:Damselfly p2:Common Newt p3:Whiptail


# To facilitate this task, a function was created to break the colum in 3 elements:
df_werated_clean = df_werated_clean.apply(get_url_elements, axis=1)


df_werated_clean.columns

Index(['tweet_id', 'in_reply_to_status_id', 'in_reply_to_user_id', 'timestamp',
       'source', 'text', 'retweeted_status_id', 'retweeted_status_user_id',
       'retweeted_status_timestamp', 'expanded_urls', 'rating_numerator',
       'rating_denominator', 'name', 'doggo', 'floofer', 'pupper', 'puppo',
       'rating', 'tweet_source', 'url_ref', 'rel'],
      dtype='object')


df_werated_clean.head()


df_werated_clean.sample(5)


df_werated_clean[["doggo", "floofer", "pupper", "puppo"]] = df_werated_clean[
    ["doggo", "floofer", "pupper", "puppo"]
].replace("None", "")


df_werated_clean["dog_class"] = df_werated_clean[
    ["doggo", "floofer", "pupper", "puppo"]
].values.tolist()
df_werated_clean["dog_class"] = df_werated_clean["dog_class"].apply(
    lambda x: " & ".join(" ".join(x).split())
)


df_werated_clean.dog_class.value_counts()

                   1779
pupper              222
doggo                72
puppo                23
doggo & pupper       10
floofer               9
doggo & puppo         1
doggo & floofer       1
Name: dog_class, dtype: int64


df_werated_clean["dog_stage"] = df_werated_clean["dog_class"].str.extract(
    "(doggo|floofer|pupper|puppo)"
)


df_werated_clean["dog_stage"] = df_werated_clean["dog_stage"].fillna("undefined")


df_werated_clean.dog_stage.value_counts()

undefined    1779
pupper        222
doggo          84
puppo          23
floofer         9
Name: dog_stage, dtype: int64


df_werated_clean.sample(5)


df_img_pred_clean.sample()


df_img_pred_clean.drop(
    columns=[
        "p1",
        "p1_conf",
        "p1_dog",
        "p2",
        "p2_conf",
        "p2_dog",
        "p3",
        "p3_conf",
        "p3_dog",
    ],
    inplace=True,
)


df_werated_clean.sample()


# Discarding more unnecessary columns
df_werated_clean.drop(
    columns=[
        "in_reply_to_status_id",
        "in_reply_to_user_id",
        "retweeted_status_id",
        "retweeted_status_user_id",
        "retweeted_status_timestamp",
        "source",
        "doggo",
        "floofer",
        "pupper",
        "puppo",
        "dog_class",
    ],
    inplace=True,
)


df_werated_clean = df_werated_clean[
    [
        "tweet_id",
        "timestamp",
        "text",
        "expanded_urls",
        "rating_numerator",
        "rating_denominator",
        "rating",
        "name",
        "tweet_source",
        "url_ref",
        "rel",
        "dog_stage",
    ]
]


df_tweet_details_clean.drop(columns=["source"], inplace=True)


df_img_pred_clean


df_werated_clean.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2117 entries, 0 to 2116
Data columns (total 12 columns):
 #   Column              Non-Null Count  Dtype         
---  ------              --------------  -----         
 0   tweet_id            2117 non-null   object        
 1   timestamp           2117 non-null   datetime64[ns]
 2   text                2117 non-null   object        
 3   expanded_urls       2117 non-null   object        
 4   rating_numerator    2117 non-null   float64       
 5   rating_denominator  2117 non-null   float64       
 6   rating              2117 non-null   float64       
 7   name                2117 non-null   object        
 8   tweet_source        2117 non-null   object        
 9   url_ref             2117 non-null   object        
 10  rel                 2117 non-null   object        
 11  dog_stage           2117 non-null   object        
dtypes: datetime64[ns](1), float64(3), object(8)
memory usage: 198.6+ KB


df_werated_clean.sample(10)


df_tweet_details_clean.sample(10)


df_weratedogs_all = df_werated_clean.merge(
    df_img_pred_clean, on="tweet_id", how="inner"
).merge(df_tweet_details_clean, on="tweet_id", how="inner")


df_weratedogs_all.columns

Index(['tweet_id', 'timestamp', 'text', 'expanded_urls', 'rating_numerator',
       'rating_denominator', 'rating', 'name', 'tweet_source', 'url_ref',
       'rel', 'dog_stage', 'jpg_url', 'img_num', 'pred_dog_breed', 'pred_conf',
       'retweet_count', 'favorite_count', 'followers_count', 'friends_count',
       'retweeted_status'],
      dtype='object')


# Changing some column names to a more meninful name
df_weratedogs_all = df_weratedogs_all.rename(
    columns={
        "text": "tweet_text",
        "expanded_urls": "tweet_url",
        "rating_numerator": "dog_rating_numerator",
        "rating_denominator": "dog_rating_denominator",
        "rating": "dog_rating",
        "name": "dog_name",
    }
)


df_weratedogs_all.columns

Index(['tweet_id', 'timestamp', 'tweet_text', 'tweet_url',
       'dog_rating_numerator', 'dog_rating_denominator', 'dog_rating',
       'dog_name', 'tweet_source', 'url_ref', 'rel', 'dog_stage', 'jpg_url',
       'img_num', 'pred_dog_breed', 'pred_conf', 'retweet_count',
       'favorite_count', 'followers_count', 'friends_count',
       'retweeted_status'],
      dtype='object')


df_weratedogs_all.sample(5)


df_weratedogs_all.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1994 entries, 0 to 1993
Data columns (total 21 columns):
 #   Column                  Non-Null Count  Dtype         
---  ------                  --------------  -----         
 0   tweet_id                1994 non-null   object        
 1   timestamp               1994 non-null   datetime64[ns]
 2   tweet_text              1994 non-null   object        
 3   tweet_url               1994 non-null   object        
 4   dog_rating_numerator    1994 non-null   float64       
 5   dog_rating_denominator  1994 non-null   float64       
 6   dog_rating              1994 non-null   float64       
 7   dog_name                1994 non-null   object        
 8   tweet_source            1994 non-null   object        
 9   url_ref                 1994 non-null   object        
 10  rel                     1994 non-null   object        
 11  dog_stage               1994 non-null   object        
 12  jpg_url                 1994 non-null   object        
 13  img_num                 1994 non-null   int64         
 14  pred_dog_breed          1994 non-null   object        
 15  pred_conf               1686 non-null   float64       
 16  retweet_count           1994 non-null   int64         
 17  favorite_count          1994 non-null   int64         
 18  followers_count         1994 non-null   int64         
 19  friends_count           1994 non-null   int64         
 20  retweeted_status        1994 non-null   object        
dtypes: datetime64[ns](1), float64(4), int64(5), object(11)
memory usage: 342.7+ KB


df_weratedogs_all.to_csv(
    "twitter_archive_master.csv", encoding="utf-8", index=False, sep=";"
)


# as per documentation in https://pandas.pydata.org/pandas-docs/version/1.3/reference/api/pandas.Grouper.html, will
# use pd.Grouper to group counts of Tweets per month

df_tweets_per_month = (
    df_weratedogs_all.groupby(pd.Grouper(key="timestamp", freq="M"))
    .count()
    .reset_index()[["timestamp", "tweet_id"]]
)


plt.figure(figsize=(8, 8))
plt.xlabel("Month-Year")
plt.ylabel("Number of Tweets")

plt.plot(df_tweets_per_month["timestamp"], df_tweets_per_month["tweet_id"])
plt.grid(visible=True, which="major", axis="both")
plt.title("Tweets per Month");


df_avg_followers = (
    df_weratedogs_all.groupby(pd.Grouper(key="timestamp", freq="M"))
    .mean()
    .reset_index()
)
df_avg_followers = df_avg_followers[["timestamp", "followers_count", "retweet_count"]]
df_avg_followers.followers_count = df_avg_followers.followers_count.astype(int)


df_tweet_ct_followers = df_tweets_per_month.merge(df_avg_followers, on="timestamp")
df_tweet_ct_followers.rename(columns={"tweet_id": "tweet_count"}, inplace=True)
df_tweet_ct_followers.corr()


z = np.polyfit(
    df_tweet_ct_followers["tweet_count"], df_tweet_ct_followers["followers_count"], 1
)
p = np.poly1d(z)
plt.xlabel("Tweet Count")
plt.ylabel("Followers Average")
plt.title("Linear Correlation Between Tweet Counts and Followers Average", y=1.08)

plt.scatter(
    x=df_tweet_ct_followers["tweet_count"],
    y=df_tweet_ct_followers["followers_count"],
    marker="^",
)
plt.plot(
    df_tweet_ct_followers["tweet_count"], p(df_tweet_ct_followers["tweet_count"]), "r--"
);


# Let's discard the uncatalogued breeds
df_only_dogs = df_weratedogs_all.query("pred_dog_breed != 'Not a Dog'")


print(f"{df_only_dogs['pred_dog_breed'].nunique()} Distinct predicted breeds at an average confidence level of {df_only_dogs['pred_conf'].mean()}")

113 Distinct predicted breeds at an average confidence level of 0.5499357432711743


df_only_dogs.pred_dog_breed.value_counts().head(10)

Golden Retriever      158
Labrador Retriever    108
Pembroke               95
Chihuahua              91
Pug                    62
Toy Poodle             51
Chow                   48
Samoyed                42
Pomeranian             42
Malamute               33
Name: pred_dog_breed, dtype: int64


df_only_dogs_name = df_only_dogs.query("dog_name != 'None'")
df_only_dogs_name.dog_name.value_counts().head(10)

Lucy       10
Charlie    10
Cooper     10
Tucker      9
Oliver      9
Penny       8
Daisy       7
Winston     7
Sadie       7
Jax         6
Name: dog_name, dtype: int64


plt.title("Top 10 Most Commom Dog Names")
plt.xlabel("Number of Records", c = 'r')
plt.ylabel("Dog Name", c = 'r')
df_only_dogs_name.dog_name.value_counts().head(10).plot(kind = 'barh');

	rating_numerator	rating_denominator
count	2356.000000	2356.000000
mean	13.126486	10.455433
std	45.876648	6.745237
min	0.000000	0.000000
25%	10.000000	10.000000
50%	11.000000	10.000000
75%	12.000000	10.000000
max	1776.000000	170.000000

	tweet_id	img_num	p1_conf	p2_conf	p3_conf
count	2.075000e+03	2075.000000	2075.000000	2.075000e+03	2.075000e+03
mean	7.384514e+17	1.203855	0.594548	1.345886e-01	6.032417e-02
std	6.785203e+16	0.561875	0.271174	1.006657e-01	5.090593e-02
min	6.660209e+17	1.000000	0.044333	1.011300e-08	1.740170e-10
25%	6.764835e+17	1.000000	0.364412	5.388625e-02	1.622240e-02
50%	7.119988e+17	1.000000	0.588230	1.181810e-01	4.944380e-02
75%	7.932034e+17	1.000000	0.843855	1.955655e-01	9.180755e-02
max	8.924206e+17	4.000000	1.000000	4.880140e-01	2.734190e-01

	tweet_id	retweet_count	favorite_count	followers_count	friends_count
count	2.354000e+03	2354.000000	2354.000000	2.354000e+03	2354.0
mean	7.426978e+17	3164.797366	8080.968564	3.200942e+06	104.0
std	6.852812e+16	5284.770364	11814.771334	4.457302e+01	0.0
min	6.660209e+17	0.000000	0.000000	3.200799e+06	104.0
25%	6.783975e+17	624.500000	1415.000000	3.200898e+06	104.0
50%	7.194596e+17	1473.500000	3603.500000	3.200945e+06	104.0
75%	7.993058e+17	3652.000000	10122.250000	3.200953e+06	104.0
max	8.924206e+17	79515.000000	132810.000000	3.201018e+06	104.0

	tweet_id	rating_numerator	rating_denominator	rating
421	807106840509214720	13.0	10.0	1.3
1745	672984142909456390	9.0	10.0	0.9
1530	678767140346941444	8.0	10.0	0.8
1817	671488513339211776	8.0	10.0	0.8
8	890609185150312448	13.0	10.0	1.3
1861	670764103623966721	10.0	10.0	1.0
77	875144289856114688	13.0	10.0	1.3
1324	689289219123089408	13.0	10.0	1.3
1142	703407252292673536	10.0	10.0	1.0
439	802952499103731712	12.0	10.0	1.2
513	789903600034189313	13.0	10.0	1.3
1815	671504605491109889	11.0	10.0	1.1
1572	677187300187611136	9.0	10.0	0.9
1125	704871453724954624	12.0	10.0	1.2
38	884247878851493888	13.0	10.0	1.3
1264	693262851218264065	11.0	10.0	1.1
1712	673707060090052608	10.0	10.0	1.0
1480	680805554198020098	9.0	10.0	0.9
940	725786712245440512	11.0	10.0	1.1
636	770069151037685760	10.0	10.0	1.0

	rating_numerator	rating_denominator	rating
count	2117.000000	2117.000000	2117.000000
mean	12.187898	10.481814	1.165293
std	40.241013	7.053755	3.944543
min	0.000000	2.000000	0.000000
25%	10.000000	10.000000	1.000000
50%	11.000000	10.000000	1.100000
75%	12.000000	10.000000	1.200000
max	1776.000000	170.000000	177.600000

	tweet_id	rating_numerator	rating_denominator
784	775096608509886464	9	11
1202	716439118184652801	50	50
1662	682962037429899265	7	11

	tweet_id	retweet_count	favorite_count	followers_count	friends_count	source	retweeted_status
0	892420643555336193	8853	39467	3200889	104	<a href="http://twitter.com/download/iphone" r...	Original Tweet
1	892177421306343426	6514	33819	3200889	104	<a href="http://twitter.com/download/iphone" r...	Original Tweet
2	891815181378084864	4328	25461	3200889	104	<a href="http://twitter.com/download/iphone" r...	Original Tweet
3	891689557279858688	8964	42908	3200889	104	<a href="http://twitter.com/download/iphone" r...	Original Tweet
4	891327558926688256	9774	41048	3200889	104	<a href="http://twitter.com/download/iphone" r...	Original Tweet

	tweet_id	in_reply_to_status_id	in_reply_to_user_id	timestamp	source	text	retweeted_status_id	retweeted_status_user_id	retweeted_status_timestamp	expanded_urls	rating_numerator	rating_denominator	name	doggo	floofer	pupper	puppo
30	886267009285017600	886266357075128320	2281181600	2017-07-15 16:51:35	<a href="http://twitter.com/download/iphone" r...	@NonWhiteHat @MayhewMayhem omg hello tanner yo...	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
55	881633300179243008	881607037314052096	47384430	2017-07-02 21:58:53	<a href="http://twitter.com/download/iphone" r...	@roushfenway These are good dogs but 17/10 is ...	NaN	NaN	NaN	NaN	17	10	None	None	None	None	None
64	879674319642796034	879553827334172672	3105440746	2017-06-27 12:14:36	<a href="http://twitter.com/download/iphone" r...	@RealKentMurphy 14/10 confirmed	NaN	NaN	NaN	NaN	14	10	None	None	None	None	None
113	870726314365509632	870726202742493184	16487760	2017-06-02 19:38:25	<a href="http://twitter.com/download/iphone" r...	@ComplicitOwl @ShopWeRateDogs >10/10 is res...	NaN	NaN	NaN	NaN	10	10	None	None	None	None	None
148	863427515083354112	863425645568774144	77596200	2017-05-13 16:15:35	<a href="http://twitter.com/download/iphone" r...	@Jack_Septic_Eye I'd need a few more pics to p...	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
179	857214891891077121	857156678055342080	180670967	2017-04-26 12:48:51	<a href="http://twitter.com/download/iphone" r...	@Marc_IRL pixelated af 12/10	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
186	856288084350160898	856286004109553664	279280991	2017-04-23 23:26:03	<a href="http://twitter.com/download/iphone" r...	@xianmcguire @Jenna_Marbles Kardashians wouldn...	NaN	NaN	NaN	NaN	14	10	None	None	None	None	None
188	855862651834028034	855861584463351808	194351775	2017-04-22 19:15:32	<a href="http://twitter.com/download/iphone" r...	@dhmontgomery We also gave snoop dogg a 420/10...	NaN	NaN	NaN	NaN	420	10	None	None	None	None	None
189	855860136149123072	855858535607001088	13615722	2017-04-22 19:05:32	<a href="http://twitter.com/download/iphone" r...	@s8n You tried very hard to portray this good ...	NaN	NaN	NaN	NaN	666	10	None	None	None	None	None
218	850333567704068097	850328818778820608	21955058	2017-04-07 13:04:55	<a href="http://twitter.com/download/iphone" r...	@markhoppus MARK THAT DOG HAS SEEN AND EXPERIE...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
228	848213670039564288	848212111729840128	4196983835	2017-04-01 16:41:12	<a href="http://twitter.com/download/iphone" r...	Jerry just apuppologized to me. He said there ...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
234	847617282490613760	847606175596138496	4196983835	2017-03-31 01:11:22	<a href="http://twitter.com/download/iphone" r...	.@breaannanicolee PUPDATE: Cannon has a heart ...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
274	840698636975636481	840698300298862592	840547864354918400	2017-03-11 22:59:09	<a href="http://twitter.com/download/iphone" r...	@0_kelvin_0 >10/10 is reserved for puppos s...	NaN	NaN	NaN	NaN	10	10	None	None	None	None	None
290	838150277551247360	838145498691194880	21955058	2017-03-04 22:12:52	<a href="http://twitter.com/download/iphone" r...	@markhoppus 182/10	NaN	NaN	NaN	NaN	182	10	None	None	None	None	None
291	838085839343206401	838085539362402304	2894131180	2017-03-04 17:56:49	<a href="http://twitter.com/download/iphone" r...	@bragg6of8 @Andy_Pace_ we are still looking fo...	NaN	NaN	NaN	NaN	15	10	None	None	None	None	None
313	835246439529840640	835245984028504064	26259576	2017-02-24 21:54:03	<a href="http://twitter.com/download/iphone" r...	@jonnysun @Lin_Manuel ok jomny I know you're e...	NaN	NaN	NaN	NaN	960	0	None	None	None	None	None
342	832088576586297345	832087547559997440	30582082	2017-02-16 04:45:50	<a href="http://twitter.com/download/iphone" r...	@docmisterio account started on 11/15/15	NaN	NaN	NaN	NaN	11	15	None	None	None	None	None
346	831926988323639298	831903044224835584	20683724	2017-02-15 18:03:45	<a href="http://twitter.com/download/iphone" r...	@UNC can confirm 12/10	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
375	828361771580813312	NaN	NaN	2017-02-05 21:56:51	<a href="http://twitter.com" rel="nofollow">Tw...	Beebop and Doobert should start a band 12/10 w...	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
387	826598799820865537	826598365270007808	4196983835	2017-02-01 01:11:25	<a href="http://twitter.com/download/iphone" r...	I was going to do 007/10, but the joke wasn't ...	NaN	NaN	NaN	NaN	7	10	None	None	None	None	None
409	823333489516937216	823326389336244224	1582853809	2017-01-23 00:56:15	<a href="http://twitter.com/download/iphone" r...	@HistoryInPics 13/10	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
427	821153421864615936	821152592717697024	113211856	2017-01-17 00:33:26	<a href="http://twitter.com/download/iphone" r...	@imgur for a polar bear tho I'd say 13/10 is a...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
498	813130366689148928	813127251579564032	4196983835	2016-12-25 21:12:41	<a href="http://twitter.com/download/iphone" r...	I've been informed by multiple sources that th...	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
513	811647686436880384	811627233043480576	4196983835	2016-12-21 19:01:02	<a href="http://twitter.com/download/iphone" r...	PUPDATE: I've been informed that Augie was act...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
570	801854953262350336	801854330672447488	11856342	2016-11-24 18:28:13	<a href="http://twitter.com/download/iphone" r...	.@NBCSports OMG THE TINY HAT I'M GOING TO HAVE...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
576	800859414831898624	800857954417262592	291859009	2016-11-22 00:32:18	<a href="http://twitter.com/download/iphone" r...	@SkyWilliams doggo simply protecting you from ...	NaN	NaN	NaN	NaN	11	10	None	doggo	None	None	None
611	797165961484890113	797123751162839040	29166305	2016-11-11 19:55:50	<a href="http://twitter.com/download/iphone" r...	@JODYHiGHROLLER it may be an 11/10 but what do...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
701	786051337297522688	772743019447607296	730505014150582272	2016-10-12 03:50:17	<a href="http://twitter.com/download/iphone" r...	13/10 for breakdancing puppo @shibbnbot	NaN	NaN	NaN	NaN	13	10	None	None	None	None	puppo
707	785515384317313025	NaN	NaN	2016-10-10 16:20:36	<a href="http://twitter.com/download/iphone" r...	Today, 10/10, should be National Dog Rates Day	NaN	NaN	NaN	NaN	10	10	None	None	None	None	None
843	766714921925144576	766711819364888576	4196983835	2016-08-19 19:14:16	<a href="http://twitter.com/download/iphone" r...	His name is Charley and he already has a new s...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
857	763956972077010945	763865174553964544	15846407	2016-08-12 04:35:10	<a href="http://twitter.com/download/iphone" r...	@TheEllenShow I'm not sure if you know this bu...	NaN	NaN	NaN	NaN	12	10	None	doggo	None	None	None
967	750381685133418496	750180498832404480	4717297476	2016-07-05 17:31:49	<a href="http://twitter.com/download/iphone" r...	13/10 such a good doggo\n@spaghemily	NaN	NaN	NaN	NaN	13	10	None	doggo	None	None	None
1005	747651430853525504	747648653817413632	4196983835	2016-06-28 04:42:46	<a href="http://twitter.com/download/iphone" r...	Other pupper asked not to have his identity sh...	NaN	NaN	NaN	NaN	12	10	None	None	None	pupper	None
1080	738891149612572673	738411919896285184	358972768	2016-06-04 00:32:32	<a href="http://twitter.com/download/iphone" r...	@mount_alex3 13/10	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
1295	707983188426153984	707980065892667392	2319108198	2016-03-10 17:35:20	<a href="http://twitter.com/download/iphone" r...	@serial @MrRoles OH MY GOD I listened to all o...	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
1345	704491224099647488	704485744639909888	28785486	2016-03-01 02:19:31	<a href="http://twitter.com/download/iphone" r...	13/10 hero af\n@ABC	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
1445	696518437233913856	NaN	NaN	2016-02-08 02:18:30	<a href="http://twitter.com/download/iphone" r...	Oh my god 10/10 for every little hot dog pupper	NaN	NaN	NaN	NaN	10	10	None	None	None	pupper	None
1446	696490539101908992	696488710901260288	4196983835	2016-02-08 00:27:39	<a href="http://twitter.com/download/iphone" r...	After reading the comments I may have overesti...	NaN	NaN	NaN	NaN	1	10	None	None	None	None	None
1474	693644216740769793	693642232151285760	4196983835	2016-01-31 03:57:23	<a href="http://twitter.com/download/iphone" r...	BREAKING PUPDATE: I've just been notified that...	NaN	NaN	NaN	NaN	10	10	None	None	None	None	None
1479	693582294167244802	693572215938367488	1198988510	2016-01-30 23:51:19	<a href="http://twitter.com/download/iphone" r...	Personally I'd give him an 11/10. Not sure why...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
1497	692423280028966913	692417313023332352	4196983835	2016-01-27 19:05:49	<a href="http://twitter.com/download/iphone" r...	PUPDATE: just noticed this dog has some extra ...	NaN	NaN	NaN	NaN	9	10	None	None	None	None	None
1523	690607260360429569	690341253549002752	467036706	2016-01-22 18:49:36	<a href="http://twitter.com/download/iphone" r...	12/10 @LightningHoltt	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
1598	686035780142297088	686034024800862208	4196983835	2016-01-10 04:04:10	<a href="http://twitter.com/download/iphone" r...	Yes I do realize a rating of 4/20 would've bee...	NaN	NaN	NaN	NaN	4	20	None	None	None	None	None
1605	685681090388975616	685547936038666240	4196983835	2016-01-09 04:34:45	<a href="http://twitter.com/download/iphone" r...	Jack deserves another round of applause. If yo...	NaN	NaN	NaN	NaN	14	10	None	None	None	None	None
1618	684969860808454144	684959798585110528	4196983835	2016-01-07 05:28:35	<a href="http://twitter.com/download/iphone" r...	For those who claim this is a goat, u are wron...	NaN	NaN	NaN	NaN	5	10	None	None	None	None	None
1663	682808988178739200	682788441537560576	4196983835	2016-01-01 06:22:03	<a href="http://twitter.com/download/iphone" r...	I'm aware that I could've said 20/16, but here...	NaN	NaN	NaN	NaN	20	16	None	None	None	None	None
1689	681340665377193984	681339448655802368	4196983835	2015-12-28 05:07:27	<a href="http://twitter.com/download/iphone" r...	I've been told there's a slight possibility he...	NaN	NaN	NaN	NaN	5	10	None	None	None	None	None
1774	678023323247357953	678021115718029312	4196983835	2015-12-19 01:25:31	<a href="http://twitter.com/download/iphone" r...	After getting lost in Reese's eyes for several...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
1819	676590572941893632	676588346097852416	4196983835	2015-12-15 02:32:17	<a href="http://twitter.com/download/iphone" r...	After some outrage from the crowd. Bubbles is ...	NaN	NaN	NaN	NaN	7	10	None	None	None	None	None
1844	675849018447167488	675845657354215424	4196983835	2015-12-13 01:25:37	<a href="http://twitter.com/download/iphone" r...	This dog is being demoted to a 9/10 for not we...	NaN	NaN	NaN	NaN	9	10	None	None	None	None	None
1895	674742531037511680	674739953134403584	4196983835	2015-12-10 00:08:50	<a href="http://twitter.com/download/iphone" r...	Some clarification is required. The dog is sin...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
1905	674606911342424069	674468880899788800	4196983835	2015-12-09 15:09:55	<a href="http://twitter.com/download/iphone" r...	The 13/10 also takes into account this impecca...	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
1914	674330906434379776	665814696700723200	16374678	2015-12-08 20:53:11	<a href="http://twitter.com/download/iphone" r...	13/10\n@ABC7	NaN	NaN	NaN	NaN	13	10	None	None	None	None	None
1940	673716320723169284	673715861853720576	4196983835	2015-12-07 04:11:02	<a href="http://twitter.com/download/iphone" r...	The millennials have spoken and we've decided ...	NaN	NaN	NaN	NaN	1	10	None	None	None	None	None
2038	671550332464455680	671544874165002240	4196983835	2015-12-01 04:44:10	<a href="http://twitter.com/download/iphone" r...	After 22 minutes of careful deliberation this ...	NaN	NaN	NaN	NaN	1	10	None	None	None	None	None
2149	669684865554620416	669354382627049472	4196983835	2015-11-26 01:11:28	<a href="http://twitter.com/download/iphone" r...	After countless hours of research and hundreds...	NaN	NaN	NaN	NaN	11	10	None	None	None	None	None
2189	668967877119254528	668920717132582912	21435658	2015-11-24 01:42:25	<a href="http://twitter.com/download/iphone" r...	12/10 good shit Bubka\n@wane15	NaN	NaN	NaN	NaN	12	10	None	None	None	None	None
2298	667070482143944705	667065535570550784	4196983835	2015-11-18 20:02:51	<a href="http://twitter.com/download/iphone" r...	After much debate this dog is being upgraded t...	NaN	NaN	NaN	NaN	10	10	None	None	None	None	None

	p1	p2	p3
1722	Acorn Squash	Toilet Seat	Toy Poodle
65	Toy Poodle	Chihuahua	Bluetick
250	Shetland Sheepdog	Home Theater	American Staffordshire Terrier
1954	Golden Retriever	Labrador Retriever	Tibetan Mastiff
915	Chihuahua	Pembroke	Basenji

	tweet_id	jpg_url	img_num	p1	p1_conf	p1_dog	p2	p2_conf	p2_dog	p3	p3_conf	p3_dog	pred_dog_breed	pred_conf
418	674036086168010753	https://pbs.twimg.com/media/CVqoPslWEAEk7EC.jpg	1	Toy Poodle	0.685617	True	Miniature Poodle	0.151936	True	Maltese Dog	0.045531	True	Toy Poodle	0.685617
909	700518061187723268	https://pbs.twimg.com/media/Cbi9dI_UYAAgkyC.jpg	1	American Staffordshire Terrier	0.569501	True	Staffordshire Bullterrier	0.211308	True	Chihuahua	0.121839	True	American Staffordshire Terrier	0.569501
2	666033412701032449	https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg	1	German Shepherd	0.596461	True	Malinois	0.138584	True	Bloodhound	0.116197	True	German Shepherd	0.596461
2034	883838122936631299	https://pbs.twimg.com/media/DEQGFgAXUAAEvfi.jpg	1	Doberman	0.610946	True	Miniature Pinscher	0.299603	True	Kelpie	0.063020	True	Doberman	0.610946
21	666293911632134144	https://pbs.twimg.com/media/CT8mx7KW4AEQu8N.jpg	1	Three-Toed Sloth	0.914671	False	Otter	0.015250	False	Great Grey Owl	0.013207	False	Not a Dog	NaN
596	679527802031484928	https://pbs.twimg.com/media/CW4q7jDWkAA2y8g.jpg	1	Mailbox	0.336393	False	Cannon	0.158936	False	Cuirass	0.052647	False	Not a Dog	NaN
1280	750132105863102464	https://pbs.twimg.com/media/CmkBKuwWgAAamOI.jpg	1	Toy Poodle	0.478018	True	Miniature Poodle	0.207458	True	Croquet Ball	0.085879	False	Toy Poodle	0.478018
31	666421158376562688	https://pbs.twimg.com/media/CT-aggCXAAIMfT3.jpg	1	Blenheim Spaniel	0.906777	True	Cocker Spaniel	0.090346	True	Shih-Tzu	0.001117	True	Blenheim Spaniel	0.906777
123	668226093875376128	https://pbs.twimg.com/media/CUYEFlQXAAUkPGm.jpg	1	Trombone	0.390339	False	Cornet	0.314149	False	French Horn	0.255182	False	Not a Dog	NaN
977	707038192327901184	https://pbs.twimg.com/media/Cc_ney1W4AANuY3.jpg	1	Pug	0.642426	True	Llama	0.057306	False	French Bulldog	0.054186	True	Pug	0.642426

	tweet_id	jpg_url	img_num	p1	p1_conf	p1_dog	p2	p2_conf	p2_dog	p3	p3_conf	p3_dog	pred_dog_breed	pred_conf
2046	886680336477933568	https://pbs.twimg.com/media/DE4fEDzWAAAyHMM.jpg	1	Convertible	0.738995	False	Sports Car	0.139952	False	Car Wheel	0.044173	False	Not a Dog	NaN
232	670417414769758208	https://pbs.twimg.com/media/CU3NE8EWUAEVdPD.jpg	1	Sea Urchin	0.493257	False	Porcupine	0.460565	False	Cardoon	0.008146	False	Not a Dog	NaN
112	667911425562669056	https://pbs.twimg.com/media/CUTl5m1WUAAabZG.jpg	1	Frilled Lizard	0.257695	False	Ox	0.235160	False	Triceratops	0.085317	False	Not a Dog	NaN
1057	714631576617938945	https://pbs.twimg.com/media/CerhoBWWAAA5eLL.jpg	1	Meerkat	0.143497	False	Weasel	0.117402	False	Black-Footed Ferret	0.099933	False	Not a Dog	NaN
929	702932127499816960	https://pbs.twimg.com/media/CcFRCfRW4AA5a72.jpg	1	Wallaby	0.410710	False	Wombat	0.239332	False	Beaver	0.149605	False	Not a Dog	NaN
512	676215927814406144	https://pbs.twimg.com/media/CWJmzNsWUAE706Z.jpg	1	Hamster	0.999484	False	Guinea Pig	0.000158	False	Broccoli	0.000061	False	Not a Dog	NaN
840	694669722378485760	https://pbs.twimg.com/media/CaP2bS8WYAAsMdx.jpg	2	Beaver	0.457094	False	Mongoose	0.228298	False	Marmot	0.148309	False	Not a Dog	NaN
366	672902681409806336	https://pbs.twimg.com/media/CVahaz9XAAA8uTy.jpg	1	Ram	0.374466	False	Bighorn	0.159621	False	Arabian Camel	0.111919	False	Not a Dog	NaN
446	674664755118911488	https://pbs.twimg.com/media/CVzkA7-WsAAcXz6.jpg	1	African Crocodile	0.330625	False	American Alligator	0.075356	False	Mink	0.057481	False	Not a Dog	NaN
1755	824796380199809024	https://pbs.twimg.com/media/CwiuEJmW8AAZnit.jpg	2	Gas Pump	0.676439	False	Harvester	0.049995	False	Swing	0.044660	False	Not a Dog	NaN

	tweet_id	in_reply_to_status_id	in_reply_to_user_id	timestamp	source	text	retweeted_status_id	retweeted_status_user_id	retweeted_status_timestamp	expanded_urls	...	rating_denominator	name	doggo	floofer	pupper	puppo	rating	tweet_source	url_ref	rel
0	892420643555336193	NaN	NaN	2017-08-01 16:23:56	<a href="http://twitter.com/download/iphone" r...	This is Phineas. He's a mystical boy. Only eve...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/892420643...	...	10.0	Phineas	None	None	None	None	1.3	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
1	892177421306343426	NaN	NaN	2017-08-01 00:17:27	<a href="http://twitter.com/download/iphone" r...	This is Tilly. She's just checking pup on you....	NaN	NaN	NaN	https://twitter.com/dog_rates/status/892177421...	...	10.0	Tilly	None	None	None	None	1.3	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
2	891815181378084864	NaN	NaN	2017-07-31 00:18:03	<a href="http://twitter.com/download/iphone" r...	This is Archie. He is a rare Norwegian Pouncin...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/891815181...	...	10.0	Archie	None	None	None	None	1.2	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
3	891689557279858688	NaN	NaN	2017-07-30 15:58:51	<a href="http://twitter.com/download/iphone" r...	This is Darla. She commenced a snooze mid meal...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/891689557...	...	10.0	Darla	None	None	None	None	1.3	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
4	891327558926688256	NaN	NaN	2017-07-29 16:00:24	<a href="http://twitter.com/download/iphone" r...	This is Franklin. He would like you to stop ca...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/891327558...	...	10.0	Franklin	None	None	None	None	1.2	Twitter for iPhone	http://twitter.com/download/iphone	nofollow

	tweet_id	in_reply_to_status_id	in_reply_to_user_id	timestamp	source	text	retweeted_status_id	retweeted_status_user_id	retweeted_status_timestamp	expanded_urls	...	rating_denominator	name	doggo	floofer	pupper	puppo	rating	tweet_source	url_ref	rel
1048	710153181850935296	NaN	NaN	2016-03-16 17:18:07	<a href="http://twitter.com/download/iphone" r...	This is Lacy. She's tipping her hat to you. Da...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/710153181...	...	10.0	Lacy	None	None	None	None	1.1	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
2048	667182792070062081	NaN	NaN	2015-11-19 03:29:07	<a href="http://twitter.com/download/iphone" r...	This is Timison. He just told an awful joke bu...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/667182792...	...	10.0	Timison	None	None	None	None	1.0	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
1526	678969228704284672	NaN	NaN	2015-12-21 16:04:13	<a href="http://twitter.com/download/iphone" r...	Meet Lola. She's a Metamorphic Chartreuse. Pla...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/678969228...	...	10.0	Lola	None	None	None	None	1.1	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
377	814986499976527872	NaN	NaN	2016-12-31 00:08:17	<a href="http://twitter.com/download/iphone" r...	This is Cooper. Someone attacked him with a sh...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/814986499...	...	10.0	Cooper	None	None	pupper	None	1.1	Twitter for iPhone	http://twitter.com/download/iphone	nofollow
1875	670449342516494336	NaN	NaN	2015-11-28 03:49:14	<a href="http://twitter.com/download/iphone" r...	Vibrant dog here. Fabulous tail. Only 2 legs t...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/670449342...	...	10.0	None	None	None	None	None	0.5	Twitter for iPhone	http://twitter.com/download/iphone	nofollow

	tweet_id	in_reply_to_status_id	in_reply_to_user_id	timestamp	source	text	retweeted_status_id	retweeted_status_user_id	retweeted_status_timestamp	expanded_urls	...	doggo	pupper	rating	tweet_source	url_ref	rel	dog_class	dog_stage
1705	673919437611909120	NaN	NaN	2015-12-07 17:38:09	<a href="http://twitter.com/download/iphone" r...	This is Obie. He is on guard watching for evil...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/673919437...	...		pupper	1.1	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	pupper	pupper
743	752334515931054080	NaN	NaN	2016-07-11 02:51:40	<a href="http://twitter.com/download/iphone" r...	Here's a doggo trying to catch some fish. 8/10...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/752334515...	...	doggo		0.8	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	doggo	doggo
1722	673636718965334016	NaN	NaN	2015-12-06 22:54:44	<a href="http://twitter.com/download/iphone" r...	This is a Lofted Aphrodisiac Terrier named Kip...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/673636718...	...			1.0	Twitter for iPhone	http://twitter.com/download/iphone	nofollow		undefined
685	760290219849637889	NaN	NaN	2016-08-02 01:44:48	<a href="http://twitter.com" rel="nofollow">Tw...	This is Rocco. He's doing his best. 13/10 some...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/760290219...	...			1.3	Twitter Web Client	http://twitter.com	nofollow		undefined
81	874296783580663808	NaN	NaN	2017-06-12 16:06:11	<a href="http://twitter.com/download/iphone" r...	This is Jed. He may be the fanciest pupper in ...	NaN	NaN	NaN	https://twitter.com/dog_rates/status/874296783...	...		pupper	1.3	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	pupper	pupper

	tweet_id	jpg_url	img_num	pred_dog_breed	pred_conf
0	666020888022790149	https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg	1	Welsh Springer Spaniel	0.465074
1	666029285002620928	https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg	1	Redbone	0.506826
2	666033412701032449	https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg	1	German Shepherd	0.596461
3	666044226329800704	https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg	1	Rhodesian Ridgeback	0.408143
4	666049248165822465	https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg	1	Miniature Pinscher	0.560311
...	...	...	...	...	...
2070	891327558926688256	https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg	2	Basset	0.555712
2071	891689557279858688	https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg	1	Labrador Retriever	0.168086
2072	891815181378084864	https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg	1	Chihuahua	0.716012
2073	892177421306343426	https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg	1	Chihuahua	0.323581
2074	892420643555336193	https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg	1	Not a Dog	NaN

	tweet_id	jpg_url	img_num	p1	p1_conf	p1_dog	p2	p2_conf	p2_dog	p3	p3_conf	p3_dog
0	666020888022790149	https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg	1	Welsh_springer_spaniel	0.465074	True	collie	1.566650e-01	True	Shetland_sheepdog	6.142850e-02	True
1	666029285002620928	https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg	1	redbone	0.506826	True	miniature_pinscher	7.419170e-02	True	Rhodesian_ridgeback	7.201000e-02	True
2	666033412701032449	https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg	1	German_shepherd	0.596461	True	malinois	1.385840e-01	True	bloodhound	1.161970e-01	True
3	666044226329800704	https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg	1	Rhodesian_ridgeback	0.408143	True	redbone	3.606870e-01	True	miniature_pinscher	2.227520e-01	True
4	666049248165822465	https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg	1	miniature_pinscher	0.560311	True	Rottweiler	2.436820e-01	True	Doberman	1.546290e-01	True
5	666050758794694657	https://pbs.twimg.com/media/CT5Jof1WUAEuVxN.jpg	1	Bernese_mountain_dog	0.651137	True	English_springer	2.637880e-01	True	Greater_Swiss_Mountain_dog	1.619920e-02	True
6	666051853826850816	https://pbs.twimg.com/media/CT5KoJ1WoAAJash.jpg	1	box_turtle	0.933012	False	mud_turtle	4.588540e-02	False	terrapin	1.788530e-02	False
7	666055525042405380	https://pbs.twimg.com/media/CT5N9tpXIAAifs1.jpg	1	chow	0.692517	True	Tibetan_mastiff	5.827940e-02	True	fur_coat	5.444860e-02	False
8	666057090499244032	https://pbs.twimg.com/media/CT5PY90WoAAQGLo.jpg	1	shopping_cart	0.962465	False	shopping_basket	1.459380e-02	False	golden_retriever	7.958960e-03	True
9	666058600524156928	https://pbs.twimg.com/media/CT5Qw94XAAA_2dP.jpg	1	miniature_poodle	0.201493	True	komondor	1.923050e-01	True	soft-coated_wheaten_terrier	8.208610e-02	True
10	666063827256086533	https://pbs.twimg.com/media/CT5Vg_wXIAAXfnj.jpg	1	golden_retriever	0.775930	True	Tibetan_mastiff	9.371780e-02	True	Labrador_retriever	7.242660e-02	True
11	666071193221509120	https://pbs.twimg.com/media/CT5cN_3WEAAlOoZ.jpg	1	Gordon_setter	0.503672	True	Yorkshire_terrier	1.742010e-01	True	Pekinese	1.094540e-01	True
12	666073100786774016	https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg	1	Walker_hound	0.260857	True	English_foxhound	1.753820e-01	True	Ibizan_hound	9.747050e-02	True
13	666082916733198337	https://pbs.twimg.com/media/CT5m4VGWEAAtKc8.jpg	1	pug	0.489814	True	bull_mastiff	4.047220e-01	True	French_bulldog	4.895950e-02	True
14	666094000022159362	https://pbs.twimg.com/media/CT5w9gUW4AAsBNN.jpg	1	bloodhound	0.195217	True	German_shepherd	7.825980e-02	True	malinois	7.562780e-02	True
15	666099513787052032	https://pbs.twimg.com/media/CT51-JJUEAA6hV8.jpg	1	Lhasa	0.582330	True	Shih-Tzu	1.661920e-01	True	Dandie_Dinmont	8.968830e-02	True
16	666102155909144576	https://pbs.twimg.com/media/CT54YGiWUAEZnoK.jpg	1	English_setter	0.298617	True	Newfoundland	1.498420e-01	True	borzoi	1.336490e-01	True
17	666104133288665088	https://pbs.twimg.com/media/CT56LSZWoAAlJj2.jpg	1	hen	0.965932	False	cock	3.391940e-02	False	partridge	5.206580e-05	False
18	666268910803644416	https://pbs.twimg.com/media/CT8QCd1WEAADXws.jpg	1	desktop_computer	0.086502	False	desk	8.554740e-02	False	bookcase	7.947970e-02	False
19	666273097616637952	https://pbs.twimg.com/media/CT8T1mtUwAA3aqm.jpg	1	Italian_greyhound	0.176053	True	toy_terrier	1.118840e-01	True	basenji	1.111520e-01	True
20	666287406224695296	https://pbs.twimg.com/media/CT8g3BpUEAAuFjg.jpg	1	Maltese_dog	0.857531	True	toy_poodle	6.306380e-02	True	miniature_poodle	2.558060e-02	True
21	666293911632134144	https://pbs.twimg.com/media/CT8mx7KW4AEQu8N.jpg	1	three-toed_sloth	0.914671	False	otter	1.525000e-02	False	great_grey_owl	1.320720e-02	False
22	666337882303524864	https://pbs.twimg.com/media/CT9OwFIWEAMuRje.jpg	1	ox	0.416669	False	Newfoundland	2.784070e-01	True	groenendael	1.026430e-01	True
23	666345417576210432	https://pbs.twimg.com/media/CT9Vn7PWoAA_ZCM.jpg	1	golden_retriever	0.858744	True	Chesapeake_Bay_retriever	5.478680e-02	True	Labrador_retriever	1.424090e-02	True
24	666353288456101888	https://pbs.twimg.com/media/CT9cx0tUEAAhNN_.jpg	1	malamute	0.336874	True	Siberian_husky	1.476550e-01	True	Eskimo_dog	9.341240e-02	True
25	666362758909284353	https://pbs.twimg.com/media/CT9lXGsUcAAyUFt.jpg	1	guinea_pig	0.996496	False	skunk	2.402450e-03	False	hamster	4.608630e-04	False
26	666373753744588802	https://pbs.twimg.com/media/CT9vZEYWUAAlZ05.jpg	1	soft-coated_wheaten_terrier	0.326467	True	Afghan_hound	2.595510e-01	True	briard	2.068030e-01	True
27	666396247373291520	https://pbs.twimg.com/media/CT-D2ZHWIAA3gK1.jpg	1	Chihuahua	0.978108	True	toy_terrier	9.396970e-03	True	papillon	4.576810e-03	True
28	666407126856765440	https://pbs.twimg.com/media/CT-NvwmW4AAugGZ.jpg	1	black-and-tan_coonhound	0.529139	True	bloodhound	2.442200e-01	True	flat-coated_retriever	1.738100e-01	True
29	666411507551481857	https://pbs.twimg.com/media/CT-RugiWIAELEaq.jpg	1	coho	0.404640	False	barracouta	2.714850e-01	False	gar	1.899450e-01	False
30	666418789513326592	https://pbs.twimg.com/media/CT-YWb7U8AA7QnN.jpg	1	toy_terrier	0.149680	True	papillon	1.482580e-01	True	Chihuahua	1.428600e-01	True
31	666421158376562688	https://pbs.twimg.com/media/CT-aggCXAAIMfT3.jpg	1	Blenheim_spaniel	0.906777	True	cocker_spaniel	9.034640e-02	True	Shih-Tzu	1.116870e-03	True
32	666428276349472768	https://pbs.twimg.com/media/CT-g-0DUwAEQdSn.jpg	1	Pembroke	0.371361	True	chow	2.493940e-01	True	Pomeranian	2.418780e-01	True
33	666430724426358785	https://pbs.twimg.com/media/CT-jNYqW4AAPi2M.jpg	1	llama	0.505184	False	Irish_terrier	1.041090e-01	True	dingo	6.207120e-02	False
34	666435652385423360	https://pbs.twimg.com/media/CT-nsTQWEAEkyDn.jpg	1	Chesapeake_Bay_retriever	0.184130	True	chain_saw	5.677530e-02	False	power_drill	3.676340e-02	False
35	666437273139982337	https://pbs.twimg.com/media/CT-pKmRWIAAxUWj.jpg	1	Chihuahua	0.671853	True	beagle	1.246800e-01	True	Saluki	4.409420e-02	True
36	666447344410484738	https://pbs.twimg.com/media/CT-yU5QWwAEjLX5.jpg	1	curly-coated_retriever	0.322084	True	giant_schnauzer	2.879550e-01	True	Labrador_retriever	1.663310e-01	True
37	666454714377183233	https://pbs.twimg.com/media/CT-5Bs-WUAA2JeC.jpg	1	dalmatian	0.278954	True	Labrador_retriever	2.376120e-01	True	Great_Pyrenees	1.711060e-01	True
38	666644823164719104	https://pbs.twimg.com/media/CUBl6IwVAAA9_zT.jpg	1	Ibizan_hound	0.044333	True	Pembroke	4.320930e-02	True	West_Highland_white_terrier	3.890560e-02	True
39	666649482315059201	https://pbs.twimg.com/media/CUBqKnLWwAA5OQB.jpg	1	Border_collie	0.447803	True	English_springer	1.704970e-01	True	collie	1.392060e-01	True

	tweet_id	retweet_count	favorite_count	followers_count	friends_count	retweeted_status
1742	679158373988876288	9193	23568	3200953	104	Original Tweet
957	751251247299190784	6695	13791	3200943	104	Original Tweet
1266	709556954897764353	1204	3593	3200947	104	Original Tweet
1598	686007916130873345	472	2704	3200950	104	Original Tweet
471	816450570814898180	9366	33961	3200895	104	Original Tweet
1023	746369468511756288	1854	6637	3200943	104	Original Tweet
54	881633300179243008	7	129	3200890	104	Original Tweet
2221	668297328638447616	319	656	3201015	104	Original Tweet
1755	678764513869611008	544	1788	3200953	104	Original Tweet
988	748705597323898880	1067	3047	3200943	104	Original Tweet

	tweet_count	followers_count	retweet_count
tweet_count	1.000000	0.834819	-0.651202
followers_count	0.834819	1.000000	-0.859924
retweet_count	-0.651202	-0.859924	1.000000

	tweet_id	timestamp	text	expanded_urls	rating_numerator	rating_denominator	rating	name	tweet_source	url_ref	rel	dog_stage
465	797236660651966464	2016-11-12 00:36:46	This is Pancake. She loves Batman and winks li...	https://twitter.com/dog_rates/status/797236660...	12.0	10.0	1.2	Pancake	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
1944	669006782128353280	2015-11-24 04:17:01	This is Tucker. He is 100% ready for the sport...	https://twitter.com/dog_rates/status/669006782...	12.0	10.0	1.2	Tucker	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
692	759557299618865152	2016-07-31 01:12:26	This is Emma. She can't believe her last guess...	https://twitter.com/dog_rates/status/759557299...	10.0	10.0	1.0	Emma	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
1512	679527802031484928	2015-12-23 05:03:47	This little pupper just arrived. 11/10 would s...	https://twitter.com/dog_rates/status/679527802...	11.0	10.0	1.1	None	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	pupper
338	821107785811234820	2017-01-16 21:32:06	Here's a doggo who looks like he's about to gi...	https://twitter.com/dog_rates/status/821107785...	11.0	10.0	1.1	None	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	doggo
768	750041628174217216	2016-07-04 19:00:33	This is Beau. He's trying to keep his daddy fr...	https://twitter.com/dog_rates/status/750041628...	13.0	10.0	1.3	Beau	TweetDeck	https://about.twitter.com/products/tweetdeck	nofollow	undefined
303	827324948884643840	2017-02-03 01:16:53	This is Ralphie. He's being treated for an ove...	https://twitter.com/dog_rates/status/827324948...	12.0	10.0	1.2	Ralphie	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
1231	696405997980676096	2016-02-07 18:51:43	This is Berb. He just found out that they have...	https://twitter.com/dog_rates/status/696405997...	7.0	10.0	0.7	Berb	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
673	761745352076779520	2016-08-06 02:06:59	Guys.. we only rate dogs. Pls don't send any m...	https://twitter.com/dog_rates/status/761745352...	11.0	10.0	1.1	None	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined
1108	706291001778950144	2016-03-06 01:31:11	When you're just relaxin and having a swell ti...	https://twitter.com/dog_rates/status/706291001...	11.0	10.0	1.1	None	Twitter for iPhone	http://twitter.com/download/iphone	nofollow	undefined

	tweet_id	timestamp	tweet_text	tweet_url	dog_rating_numerator	dog_rating_denominator	dog_rating	dog_name	tweet_source	url_ref	...	dog_stage	jpg_url	img_num	pred_dog_breed	pred_conf	retweet_count	favorite_count	followers_count	friends_count	retweeted_status
1006	707969809498152960	2016-03-10 16:42:10	Meet Rufus. He's a Honeysuckle Firefox. Curly ...	https://twitter.com/dog_rates/status/707969809...	11.0	10.0	1.1	Rufus	Twitter for iPhone	http://twitter.com/download/iphone	...	undefined	https://pbs.twimg.com/media/CdM2xRpXEAUsR4k.jpg	1	Toy Poodle	0.908491	1097	2941	3200947	104	Original Tweet
470	793135492858580992	2016-10-31 17:00:11	Your favorite squad is looking extra h*ckin sp...	https://twitter.com/dog_rates/status/793135492...	13.0	10.0	1.3	None	Twitter for iPhone	http://twitter.com/download/iphone	...	undefined	https://pbs.twimg.com/media/CwHIg61WIAApnEV.jpg	1	Not a Dog	NaN	2893	7214	3200901	104	Original Tweet
427	800513324630806528	2016-11-21 01:37:04	This is Chef. Chef loves everyone and wants ev...	https://twitter.com/dog_rates/status/800513324...	11.0	10.0	1.1	Chef	Twitter for iPhone	http://twitter.com/download/iphone	...	undefined	https://pbs.twimg.com/media/Cxv-nkJUoAAhzMt.jpg	1	Pembroke	0.828904	3495	14685	3200898	104	Original Tweet
1684	671544874165002241	2015-12-01 04:22:29	Interesting dog here. Very large. Purple. Mani...	https://twitter.com/dog_rates/status/671544874...	6.0	10.0	0.6	None	Twitter for iPhone	http://twitter.com/download/iphone	...	undefined	https://pbs.twimg.com/media/CVHOgDvU4AAfrXD.jpg	1	Not a Dog	NaN	1145	2126	3201006	104	Original Tweet
1990	666044226329800704	2015-11-16 00:04:52	This is a purebred Piers Morgan. Loves to Netf...	https://twitter.com/dog_rates/status/666044226...	6.0	10.0	0.6	None	Twitter for iPhone	http://twitter.com/download/iphone	...	undefined	https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg	1	Rhodesian Ridgeback	0.408143	147	311	3201018	104	Original Tweet

Project: Wrangling and Analyze Data¶

Importing libraries¶

Creating Used Functions¶

Data Gathering¶

Assessing Data¶

Visual Assessment¶

Programatic Assessment¶

Quality issues¶

Due to Completeness¶

Due to Validity¶

Due to Accurrancy¶

Due to Consistency¶

Tidiness issues¶

Cleaning Data¶

Due to Quality Issues¶

Issue #1: Correcting data types¶

Define:¶

Code¶

Test¶

Issue #2: Retweets in the dataset¶

Define¶

Code¶

Test¶

Issue #3a: rows missing expanded urls¶

Define¶

Code¶

Test¶

Issue #3b: Handle the remaining rows with missing expanded urls¶

Define¶

Code¶

Test¶

Issue #4: Strange Names and Typos¶

Define¶

Code¶

Test¶

Issue #5: Incorrect ratings¶

Define¶

Code¶

Test¶

Issue #6: Correct ratings (numerator and denominator)¶

Define¶

Code¶

Test¶

Issue #7: Standardize dog breeds¶

Define¶

Code¶

Test¶

Issue #8: Identify the predicted breed¶

Define¶

Code¶

Test¶

Due to Tidness¶

Issue #1: Column Source showing more than one value¶

Define¶

Code¶

Test¶

Issue #2: Dog stage in one single column¶

Define¶

Code¶

Test¶

Issue #3: Unnecessary Columns¶

Define¶

Code¶

Test¶

Issue #4: Merge Tables¶

Define¶

Code¶

Test¶

Storing Data¶

Analyzing and Visualizing Data¶

Insights:¶

Checking the total number of tweets per month¶

Checking Correlation between the amount of tweets and follower counts¶

Number of Catalogued Breeds Followed by Confidence Level¶

10 Most Predicted Breeds¶

10 Most Common Names¶

Visualization¶